﻿using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Net;
using System.Collections;
using System.Text.RegularExpressions;
using System.Threading;

namespace SimpleCrawlerApp
{
    [Serializable]
    public class Crawler
    {
       public static StringBuilder str = new StringBuilder();
       public static StringBuilder str1 = new StringBuilder();

        private Hashtable urls = new Hashtable();
        private int count = 0;

        public static void Main(string[] args)
        {
            Crawler myCrawler = new Crawler();

            string startUrl = "http://www.cnblogs.com/dstang2000/";
            if (args.Length >= 1) startUrl = args[0];

            myCrawler.urls.Add(startUrl, false);

            new Thread(myCrawler.Crawl).Start();
        }

        private void Crawl()
        {
            str.Clear();
            str1.Clear();
            str.Append("开始爬行了……\n");
            Console.WriteLine("开始爬行了……");
            while (true)
            {
                string current = null;
                foreach (string url in urls.Keys)
                {
                    if ((bool)urls[url]) continue;
                    current = url;
                }
                if (current == null || count > 10) break;
 
                Console.WriteLine("爬行" + current + "页面");
                string html = Download(current);
                if (Regex.IsMatch(html, @".*</html>$"))
                    str.Append("爬行" + current + "页面");
                else
                    str1.Append("爬行" + current + "页面");

                urls[current] = true;
                if (Regex.IsMatch(html, @".*</html>$"))
                {
                    count++;
                    Parse(html);
                }
            }
            Console.WriteLine(count);
            Console.WriteLine("爬行结束");
        }

        public string Download(string url)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);

                string fileName = count.ToString();
                File.WriteAllText(fileName, html, Encoding.UTF8);
                return html;
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                return "";
            }
        }

        public void Parse(string html)
        {
            string strRef = @"(href|HERF)[]*=[]*[""'][^""'#>]+[""']";
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\"', '#', '>');
                if (strRef.Length == 0) continue;
                if (urls[strRef] == null)
                    urls[strRef] = false;
            }
        }
    }
}
